keep = ls()

###################################
#Load Master Key/Dykstra Crosswalk#
###################################

#Load key linking townships over time
master_key = fread('./raw/IA_WI_TOWNSHIPS_MASTER_KEY.csv')

#Unlist parent townships
master_list = master_key[, list(State, state_id, County, county_id, Township, township_id, Parent_Township)] %>% unique %>%
      .[, list( parent_tag = Parent_Township %>% str_split("; ?") %>% unlist), by = list(State, state_id, County, county_id, Township, township_id)]
master_list[, parent_name := parent_tag %>% str_extract("^[^:]+")]
master_list[, parent_year := parent_tag %>% str_extract("\\d+$")]
master_list[is.na(parent_name), parent_name := Township]
master_list[is.na(parent_year), parent_year := 1850]

#Merge in parent township IDs
parent_ids = master_key[, list(State, County, parent_name = Township, parent_id = township_id)] %>% unique

setkey(master_list, State, County, parent_name)
setkey(parent_ids, State, County, parent_name)

master_list = parent_ids[master_list] 

#Create network of townships using parent-child links as edges
edges = master_list[parent_year >= 1857 & ((State == "IOWA" & parent_year <=1868) | (State == "WISCONSIN" & parent_year <=1865)), list(to = township_id, from = parent_id)]

#add Green Lake/Marquette County WI,  edges
add_edges = c(22201,	23302,
22201,	23302,
22201,	23302,
22202,	23303,
22203,	23313,
22205,	23307,
22206,	23310,
22207,	23311,
22210,	23320,
22212,	23320,
23302,	22201,
23303,	22202,
23307,	22205,
23310,	22206,
23311,	22207,
23313,	22203,
23301, 22208,
22208, 23301) %>% 
  matrix(byrow = T, ncol = 2) %>%
  as.data.table %>% unique %>%
  setnames(paste0("V", 1:2), c('to', 'from'))

edges = rbind(edges, add_edges)

vertices = data.table(master_list$township_id %>% unique)

township_graph = edges %>% graph_from_data_frame(directed = F, vertices = vertices)

#Get clusters of townships that share boundaries
groups = clusters(township_graph)

group_list = split(names(groups$membership), groups$membership)

township2cluster = mapply(function(x,y) data.table(township_id = as.numeric(x), township_cluster = y), 
                       group_list, names(group_list), SIMPLIFY = F) %>%
  rbindlist()


######
#Iowa#
######

#Subset master list to Iowa
iowa_key = master_key[State %in% "IOWA", list(State, County, Township, township_id)] %>% unique

setkey(iowa_key, township_id)
setkey(township2cluster, township_id)
iowa_key = township2cluster[iowa_key]

#Dykstra Iowa Township Returns: Township ID crosswalk
dykstra_xwalk =  fread('./raw/dykstra_township_crosswalk.csv')
dykstra_xwalk = dykstra_xwalk[, list(township_id = locality_id %>% str_split("; ?") %>% unlist %>% str_sub(1,-2) %>% as.numeric() %>% unique) , 
                              by = list(dykstra_county_name, dykstra_township_name, dykstra_county_code, dykstra_township_code)]

#Merge  Dykstra Townshipsto Township Cluster
setkey(iowa_key, township_id)
setkey(dykstra_xwalk, township_id)
iowa_elections = dykstra_xwalk[iowa_key]

#Merge in Dykstra Election Data
###############################

#counties election data
dykstra_counties = read.dta('./raw/ICPSR_04284/DS0001/04284-0001-Data.dta') %>% 
                    as.data.table %>%
                    .[, list(County = COID %>% str_to_upper(),
                             county_yes_57 = Y57BS, 
                             county_no_57 = NB57BS, 
                             county_ro_57 = NRO57BS, 
                             county_nv_57 = NV57BS, 
                             county_eev_57 = ifelse(!is.na(EEV57A), EEV57A, EEV57O),
                             county_eev_68 = EEV68, 
                             county_yes_68 = Y68BS, 
                             county_no_68 = N68BS, 
                             county_ro_68 = RO68BS, 
                             county_nv_68 = NV68BS)]
dykstra_counties[County %in% "HUMBOLT", County := "HUMBOLDT"]

#townships election data
dykstra_townships = read.dta('./raw/ICPSR_04284/DS0003/04284-0003-Data.dta', convert.factors = F) %>% 
  as.data.table %>%
  .[!is.na(Y57S) & !is.na(Y68S), 
      list(dykstra_county_code = COUNTY,
           dykstra_township_code = TNSHIP,
           township_yes_57 = Y57S, 
           township_no_57 = NB57S, 
           township_ro_57 = NRO57S, 
           township_nv_57 = NV57S, 
           township_eev_56 = EV56, 
           township_eev_70 = EV70, 
           township_yes_68 = Y68S, 
           township_no_68 = N68S, 
           township_ro_68 = RO68S, 
           township_nv_68 = NV68S)]

#merge in counties
setkey(iowa_elections, County)
setkey(dykstra_counties, County)

iowa_elections = dykstra_counties[iowa_elections]

#merge in townships
setkey(iowa_elections, dykstra_county_code, dykstra_township_code)
setkey(dykstra_townships, dykstra_county_code, dykstra_township_code)

iowa_elections = dykstra_townships[iowa_elections]

#Collapse to township/sub-county/county units
#############################################


#collapse by unique unit (township/township group/county/county subset)
iowa_elections = iowa_elections[, list(township_yes_57 = township_yes_57 %>% unique,
                                       township_no_57 = township_no_57 %>% unique,
                                       township_ro_57 = township_ro_57 %>% unique,
                                       township_nv_57 = township_nv_57 %>% unique,
                                       township_eev_56 = township_eev_56 %>% unique,
                                       township_eev_70 = township_eev_70 %>% unique,
                                       township_yes_68 = township_yes_68 %>% unique,
                                       township_no_68 = township_no_68 %>% unique,
                                       township_ro_68 = township_ro_68 %>% unique,
                                       township_nv_68 = township_nv_68 %>% unique,
                                       county_yes_57 = county_yes_57 %>% unique,
                                       county_no_57 = county_no_57 %>% unique,
                                       county_ro_57 = county_ro_57 %>% unique,
                                       county_nv_57 = county_nv_57 %>% unique,
                                       county_eev_57 = county_eev_57 %>% unique,
                                       county_eev_68 = county_eev_68 %>% unique,
                                       county_yes_68 = county_yes_68 %>% unique,
                                       county_no_68 = county_no_68 %>% unique,
                                       county_ro_68 = county_ro_68 %>% unique,
                                       county_nv_68 = county_nv_68 %>% unique,
                                       township_list = list(Township),
                                       township_id_list = list(township_id)
                                       )
                                , by = list(County, dykstra_county_code, dykstra_township_code)]


#sum township votes by county for unique dykstra county-town ids
township_vars = names(dykstra_townships) %>% str_detect("^township") %>% names(dykstra_townships)[.]
iowa_elections[,  paste0("total_", township_vars) := lapply(.SD, function(x) sum(x, na.rm = T)) , by = County, .SDcols = township_vars]

#subtract township specific-values from county totals
county_vars = names(dykstra_counties) %>% str_detect("^county") %>% names(dykstra_counties)[.]
iowa_elections[, c(county_vars) := list(
                                            county_yes_57 - total_township_yes_57,
                                            county_no_57 - total_township_no_57, 
                                            county_ro_57 - total_township_ro_57, 
                                            county_nv_57 - total_township_nv_57,
                                            county_eev_57 - total_township_eev_56,
                                            county_eev_68 - total_township_eev_70,
                                            county_yes_68 - total_township_yes_68,
                                            county_no_68 - total_township_no_68,
                                            county_ro_68 - total_township_ro_68,
                                            county_nv_68 - total_township_nv_68)]

#Create 224-Unit Panel
iowa_elections[, is_county := is.na(dykstra_township_code)]
iowa_elections[, num_county := .N, by = County]

iowa_elections = iowa_elections[, list(County, dykstra_county_code, dykstra_township_code, township_list, township_id_list,
                                       ia_unit_id = 1:.N,
                                       unit_type = ifelse((is_county), ifelse(num_county == 1, "County: Whole", "County: Part"), "Township"),
                                       suff_yes_57 = ifelse((is_county), county_yes_57, township_yes_57),
                                        suff_no_57 = ifelse((is_county), county_no_57, township_no_57),
                                        suff_ro_57 = ifelse((is_county), county_ro_57, township_ro_57),
                                        suff_nv_57 = ifelse((is_county), county_nv_57, township_nv_57),
                                        suff_eev_57 = ifelse((is_county), county_eev_57, township_eev_56),
                                        suff_yes_68 = ifelse((is_county), county_yes_68, township_yes_68),
                                        suff_no_68 = ifelse((is_county), county_no_68, township_no_68),
                                        suff_ro_68 = ifelse((is_county), county_ro_68, township_ro_68),
                                        suff_nv_68 = ifelse((is_county), county_nv_68, township_nv_68),
                                        suff_eev_68 = ifelse((is_county), county_eev_68, township_eev_70)
                      )]

#Create township to township cluster id crosswalk
iowa_elections_tp_xwalk = iowa_elections[, list(
                                                tp_name = unlist(township_list),
                                                tp_id = unlist(township_id_list)
                                                ), 
                                           by = list(ia_unit_id, County)]
fwrite(iowa_elections_tp_xwalk, './cleaned/iowa_elections_tp_xwalk.csv')
fwrite(iowa_elections, './cleaned/iowa_elections_tp_file.csv')

###########
#Wisconsin#
###########

#Subset master list to Wisconsin
wisconsin_key = master_key[State %in% "WISCONSIN", list(State, County, Township, township_id)] %>% unique

setkey(wisconsin_key, township_id)
setkey(township2cluster, township_id)
wisconsin_key = township2cluster[wisconsin_key]

#Wisconsin XWalk
mcmanus_xwalk =  fread('./raw/mcmanus_township_crosswalk.csv')
mcmanus_xwalk = mcmanus_xwalk[, list(township_id = township_id %>% str_split("; ?") %>% unlist %>% as.numeric() %>% unique) , by = list(County, Township)] %>%
                .[, list(mcmanus_county = County, mcmanus_township = Township, township_id)]


#merge Township Clusters to McManus crosswalk:
setkey(wisconsin_key, township_id)
setkey(mcmanus_xwalk, township_id)

wisconsin_elections = mcmanus_xwalk[wisconsin_key]

#Fill empty counties
wisconsin_elections[, mcmanus_county_na := mcmanus_county %>% table %>% sort(decreasing = T) %>% names %>% .[1], by = County]
wisconsin_elections[is.na(mcmanus_county), mcmanus_county := mcmanus_county_na]
wisconsin_elections[, mcmanus_county_na := NULL]

#McManus Election Data
######################

#Counties
mcmanus_counties = fread("./raw/wi_ref_1857_1865_counties.csv") %>% 
                    .[, list(mcmanus_county = County, 
                             county_suff_yes_57 = ref_1857_yes,
                             county_suff_no_57 = ref_1857_no,
                             county_suff_nv_57 = ref_1857_nv,
                             county_gov_r_57 = gov_1857_R,
                             county_gov_d_57 = gov_1857_D,  
                             county_eev_57 = voters_1857,
                             county_eev_57_alt = elig_1857,
                             county_suff_yes_65 = ifelse(is.na(ref_1865_yes_alt), ref_1865_yes, ref_1865_yes_alt),
                             county_suff_no_65 = ifelse(is.na(ref_1865_no_alt), ref_1865_no, ref_1865_no_alt),
                             county_gov_r_65 = ifelse(is.na(gov_1865_R_alt), gov_1865_R, gov_1865_R_alt),
                             county_gov_d_65 = ifelse(is.na(gov_1865_D_alt), gov_1865_D, gov_1865_D_alt),
                             county_suff_nv_65 = ref_1865_nv,
                             county_eev_65 = voters_1865
                             )] %>%
                    .[!(mcmanus_county %in% c("Soldiers", ""))]


#Townships
mcmanus_townships = fread("./raw/wi_ref_1857_1865_towns.csv") %>% 
  .[, list(mcmanus_county = County, 
           mcmanus_township = Township,
           township_suff_yes_57 = ref_1857_yes,
           township_suff_no_57 = ref_1857_no,
           township_gov_r_57 = gov_1857_R,
           township_gov_d_57 = gov_1857_D,
           township_gov_r_65 = gov_1865_R,
           township_gov_d_65 = gov_1865_D,
           township_suff_yes_65 = ref_1865_yes,
           township_suff_no_65 = ref_1865_no,
           notes_1865, notes_1857
  )]

#Fix zeros
mcmanus_townships[is.na(township_suff_yes_65) & 
                    !is.na(township_suff_no_65), 
                  township_suff_yes_65 := 0]
  
mcmanus_townships[!is.na(township_suff_yes_65) & 
                    is.na(township_suff_no_65), 
                  township_suff_no_65 := 0]

mcmanus_townships[is.na(township_suff_yes_57) & 
                    !is.na(township_suff_no_57), 
                  township_suff_yes_57 := 0]

mcmanus_townships[!is.na(township_suff_yes_57) & 
                    is.na(township_suff_no_57), 
                  township_suff_no_57 := 0]

#Drop townships with flagged results
mcmanus_townships[, note_flag := notes_1865 != ""]

#merge in county election returns
setkey(wisconsin_elections, mcmanus_county)
setkey(mcmanus_counties, mcmanus_county)

wisconsin_elections = mcmanus_counties[wisconsin_elections]

#merge in township election returns
setkey(wisconsin_elections, mcmanus_county, mcmanus_township)
setkey(mcmanus_townships, mcmanus_county, mcmanus_township)

wisconsin_elections = mcmanus_townships[wisconsin_elections]


#Collapse to township/sub-county/county units
#############################################

#collapse to township id
  #by county / township_id
  #sum of unique township vote values
  #count of unique mcmanus township values
  #list of mcmanus units used
  #
wisconsin_elections[, mcmanus_units := paste(mcmanus_county, mcmanus_township, sep = ":")]


sumNA = function(x) {
  use = na.omit(x); 
  if (length(use) > 0) {return(sum(use))} else {return(sum(x))}}

wisconsin_elections = wisconsin_elections[, list(township_suff_yes_57 = township_suff_yes_57 %>% sumNA,
                                                 township_suff_no_57 = township_suff_no_57 %>% sumNA,
                                                 township_gov_r_57 = township_gov_r_57 %>% sumNA,
                                                 township_gov_d_57 = township_gov_d_57 %>% sumNA,
                                                 township_suff_yes_65 = township_suff_yes_65 %>% sumNA,
                                                 township_suff_no_65 = township_suff_no_65 %>% sumNA,
                                                 township_gov_r_65 = township_gov_r_65 %>% sumNA,
                                                 township_gov_d_65 = township_gov_d_65 %>% sumNA,
                                                 county_suff_yes_57 = county_suff_yes_57 %>% unique,
                                                 county_suff_no_57 = county_suff_no_57 %>% unique,
                                                 county_suff_nv_57 = county_suff_nv_57 %>% unique,
                                                 county_gov_r_57 = county_gov_r_57 %>% unique,
                                                 county_gov_d_57 = county_gov_d_57 %>% unique,
                                                 county_eev_57 = county_eev_57 %>% unique,
                                                 county_eev_57_alt = county_eev_57_alt %>% unique,
                                                 county_suff_yes_65 = county_suff_yes_65 %>% unique,
                                                 county_suff_no_65 = county_suff_no_65 %>% unique,
                                                 county_suff_nv_65 = county_suff_nv_65 %>% unique,
                                                 county_gov_r_65 = county_gov_r_65 %>% unique,
                                                 county_gov_d_65 = county_gov_d_65 %>% unique,
                                                 county_eev_65 = county_eev_65 %>% unique,
                                                 mcmanus_unit_list = list(mcmanus_units),
                                                 note_flag = any(note_flag, na.rm = T))
                                          , by = list(County, Township, township_id, township_cluster)]


#Aggregate county sums by combined counties
#combine: marquette and green lake
#combine: ashland and lapointe (bayfield)
#combine: pepin and dunn
#combine douglas burnett and polk
wisconsin_elections[, County_Cluster := County]
wisconsin_elections[County %in% c("MARQUETTE", "GREEN LAKE"), County_Cluster := "GREEN LAKE;MARQUETTE"]
wisconsin_elections[County %in% c("ASHLAND", "BAYFIELD"), County_Cluster := "ASHLAND;BAYFIELD;LA POINTE"]
wisconsin_elections[County %in% c("DUNN", "PEPIN"), County_Cluster := "DUNN;PEPIN"]
wisconsin_elections[County %in% c("BURNETT", "DOUGLAS", "POLK"), County_Cluster := "BURNETT;DOUGLAS;POLK"]

county_vars = names(wisconsin_elections) %>% str_detect("^county") %>% names(wisconsin_elections)[.]
wisconsin_elections[, c(county_vars) := lapply(.SD, function(x) unique(x) %>% sum(na.rm = T)), by = County_Cluster, .SDcols = county_vars]

#collapse by township_cluster
wisconsin_elections[, census_units := paste(County, Township, sep = ":")]
wisconsin_elections = wisconsin_elections[!(duplicated(township_id)), list(township_suff_yes_57 = township_suff_yes_57 %>% sumNA,
                                                 township_suff_no_57 = township_suff_no_57 %>% sumNA,
                                                 township_gov_r_57 = township_gov_r_57 %>% sumNA,
                                                 township_gov_d_57 = township_gov_d_57 %>% sumNA,
                                                 township_suff_yes_65 = township_suff_yes_65 %>% sumNA,
                                                 township_suff_no_65 = township_suff_no_65 %>% sumNA,
                                                 township_gov_r_65 = township_gov_r_65 %>% sumNA,
                                                 township_gov_d_65 = township_gov_d_65 %>% sumNA,
                                                 county_suff_yes_57 = county_suff_yes_57 %>% unique,
                                                 county_suff_no_57 = county_suff_no_57 %>% unique,
                                                 county_suff_nv_57 = county_suff_nv_57 %>% unique,
                                                 county_gov_r_57 = county_gov_r_57 %>% unique,
                                                 county_gov_d_57 = county_gov_d_57 %>% unique,
                                                 county_eev_57 = county_eev_57 %>% unique,
                                                 county_eev_57_alt = county_eev_57_alt %>% unique,
                                                 county_suff_yes_65 = county_suff_yes_65 %>% unique,
                                                 county_suff_no_65 = county_suff_no_65 %>% unique,
                                                 county_suff_nv_65 = county_suff_nv_65 %>% unique,
                                                 county_gov_r_65 = county_gov_r_65 %>% unique,
                                                 county_gov_d_65 = county_gov_d_65 %>% unique,
                                                 county_eev_65 = county_eev_65 %>% unique,
                                                 mcmanus_unit_list = mcmanus_unit_list %>% unlist %>% list,
                                                 township_id_list = list(township_id),
                                                 census_unit_list = list(census_units),
                                                 note_flag = any(note_flag, na.rm = T))
                                          , by = list(County_Cluster, township_cluster)]

#sum township values by county for unique township clusters with complete data
township_vars = names(mcmanus_townships) %>% str_detect("^township_suff") %>% names(mcmanus_townships)[.]
wisconsin_elections[, full_panel := !(note_flag) & (rowSums(is.na(.SD)) %>% `==` (0)), .SDcols = township_vars]

township_vars = names(mcmanus_townships) %>% str_detect("^township") %>% names(mcmanus_townships)[.]
wisconsin_elections[,  paste0("total_", township_vars) := lapply(.SD, function(x) sum(x[(full_panel)], na.rm = T)) , by = County_Cluster, .SDcols = township_vars]

#subtract township specific-values from county totals
county_vars = township_vars %>% str_replace("township", "county")
wisconsin_elections[, c(county_vars) := list(
  county_suff_yes_57 - total_township_suff_yes_57,
  county_suff_no_57 - total_township_suff_no_57, 
  county_gov_r_57 - total_township_gov_r_57,
  county_gov_d_57 - total_township_gov_d_57,
  county_gov_r_65 - total_township_gov_r_65,
  county_gov_d_65 - total_township_gov_d_65,
  county_suff_yes_65 - total_township_suff_yes_65,
  county_suff_no_65 - total_township_suff_no_65)]

#Create Analysis Units

#counties with zero or positive remaining votes in left-over townships
#identify remaining non-panel townships
#split into two tables: (1) panel townships; (2) county parts
#collapse county parts table
#rbind two tables into one.
county_vars = names(mcmanus_counties) %>% str_detect("^county_suff") %>% names(mcmanus_counties)[.]
wisconsin_elections[, non_zero_remainder := (rowSums(.SD < 0) == 0) & (rowSums(.SD > 0) > 0), .SDcols = county_vars]


wisconsin_county_parts = wisconsin_elections[!(full_panel) & (non_zero_remainder), 
                                             list(
                                               suff_yes_1857 = county_suff_yes_57 %>% unique,
                                               suff_no_1857 = county_suff_no_57  %>% unique,
                                               gov_r_1857 = county_gov_r_57 %>% unique,
                                               gov_d_1857 = county_gov_d_57 %>% unique,
                                               suff_yes_1865 = county_suff_yes_65 %>% unique,
                                               suff_no_1865 = county_suff_no_65  %>% unique,
                                               gov_r_1865 = county_gov_r_65 %>% unique,
                                               gov_d_1865 = county_gov_d_65 %>% unique,
                                               mcmanus_unit_list = mcmanus_unit_list %>% unlist %>% list,
                                               township_id_list = township_id_list %>% unlist %>% list,
                                               census_unit_list = census_unit_list %>% unlist %>% list,
                                               unit_type = "County:Part"
                                             ),
                                             by = County_Cluster]
          
wisconsin_township =  wisconsin_elections[(full_panel), 
                                          list(
                                            County_Cluster,
                                            suff_yes_1857 = township_suff_yes_57,
                                            suff_no_1857 = township_suff_no_57,
                                            gov_r_1857 = township_gov_r_57,
                                            gov_d_1857 = township_gov_d_57,
                                            suff_yes_1865 = township_suff_yes_65,
                                            suff_no_1865 = township_suff_no_65,
                                            gov_r_1865 = township_gov_r_65,
                                            gov_d_1865 = township_gov_d_65,
                                            mcmanus_unit_list,
                                            township_id_list,
                                            census_unit_list,
                                            unit_type = "Township:Cluster"
                                            )]                                 

wisconsin_elections = rbind(wisconsin_township, wisconsin_county_parts)

setkey(wisconsin_elections, County_Cluster)
wisconsin_elections[, wi_unit_id := 1:.N]

wisconsin_elections_tp_xwalk = wisconsin_elections[, list(
  tp_name = unlist(census_unit_list),
  tp_id = unlist(township_id_list)
), 
by = list(wi_unit_id, County_Cluster)]

fwrite(wisconsin_elections_tp_xwalk, './cleaned/wisconsin_elections_tp_xwalk.csv')
fwrite(wisconsin_elections[, c(1:9,13:14)], './cleaned/wisconsin_elections_tp_file.csv')

#Cleanup
rm(list = setdiff(ls(), c(keep, 'keep')))
